# Basic Libraries
import pandas as pd      # For data manipulation and analysis
import numpy as np       # For numerical operations
import matplotlib.pyplot as plt  # For plotting and visualizations
import seaborn as sns    # For statistical data visualization

# Machine Learning and Statistical Analysis
from sklearn.decomposition import PCA       # For Principal Component Analysis
from sklearn.cluster import KMeans          # For K-means clustering
from sklearn.linear_model import LinearRegression  # For regression analysis
from sklearn.model_selection import train_test_split  # For splitting data into training and testing sets
from sklearn.preprocessing import StandardScaler     # For feature scaling
from sklearn.metrics import mean_squared_error, r2_score  # For evaluating regression models
from sklearn.ensemble import RandomForestRegressor   # For random forest regression
from sklearn.svm import SVR                          # For Support Vector Regression
from scipy.stats import pearsonr, ttest_ind, f_oneway  # For statistical tests

# Geospatial Libraries
import geopandas as gpd     # For geospatial data manipulation
import folium               # For interactive maps

# Advanced Data Visualization
import plotly.express as px  # For advanced interactive visualizations

# Machine Learning Interpretation
import shap                 # For model interpretation using SHAP values

# Others
import warnings
warnings.filterwarnings('ignore')  # To suppress warnings for cleaner outputs

# Load the dataset
file_path = r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\Cost_of_Living_Index_by_Country_2024.csv'
data = pd.read_csv(file_path)

# Display the first few rows of the dataset to understand its structure
data.head()

# Check for any missing values in the dataset
data.isnull().sum()

# Display summary statistics to get an overview of the data
data.describe()

# Set up the matplotlib figure
plt.figure(figsize=(18, 12))

# Plot distribution for each index
indices = ['Cost of Living Index', 'Rent Index', 'Cost of Living Plus Rent Index', 'Groceries Index', 'Restaurant Price Index', 'Local Purchasing Power Index']

for i, index in enumerate(indices):
    plt.subplot(2, 3, i+1)
    sns.histplot(data[index], kde=True, bins=30)
    plt.title(f'Distribution of {index}')
    plt.xlabel(index)
    plt.ylabel('Frequency')

plt.tight_layout()
plt.show()

# Pair plot to show relationships between indices
sns.pairplot(data[indices])
plt.suptitle('Pairwise Relationships Between Indices', y=1.02)
plt.show()

import plotly.express as px

# Create the interactive correlation matrix with light blue and light red colors
fig = px.imshow(corr_matrix, 
                text_auto=True, 
                color_continuous_scale=['#87CEFA', '#FFA07A'],  # Light blue and light red colors
                aspect='auto')

# Update the layout to center the title
fig.update_layout(
    title={
        'text': "Interactive Correlation Matrix of Indices",
        'y':0.95,
        'x':0.5,
        'xanchor': 'center',
        'yanchor': 'top'},
    title_font=dict(size=20),
    coloraxis_colorbar=dict(title="Correlation"),
)

# Show the figure
fig.show()

# Calculate Pearson correlation coefficients
correlation_matrix = data[indices].corr(method='pearson')

# Display the correlation matrix
correlation_matrix

from scipy.stats import pearsonr

# Test the significance of the correlations
p_values = data[indices].corr(method=lambda x, y: pearsonr(x, y)[1]) - np.eye(len(data[indices].columns))

# Display the p-values
p_values

# Example interpretation (to be expanded based on actual results)
if p_values.min().min() < 0.05:
    print("Several correlations between the indices are statistically significant, indicating robust relationships.")
else:
    print("The correlations observed may not be statistically significant, suggesting weaker or more complex relationships between the indices.")

Several correlations between the indices are statistically significant, indicating robust relationships.

from sklearn.cluster import KMeans
import matplotlib.pyplot as plt

# Define the features for clustering
features = data[['Cost of Living Index', 'Rent Index', 'Cost of Living Plus Rent Index', 'Groceries Index', 'Restaurant Price Index', 'Local Purchasing Power Index']]

# Determine the optimal number of clusters using the Elbow Method
sse = []
for k in range(1, 11):
    kmeans = KMeans(n_clusters=k, random_state=42)
    kmeans.fit(features)
    sse.append(kmeans.inertia_)

# Plot the results of the Elbow Method
plt.figure(figsize=(10, 6))
plt.plot(range(1, 11), sse, marker='o')
plt.title('Elbow Method For Optimal Number of Clusters')
plt.xlabel('Number of Clusters')
plt.ylabel('Sum of Squared Distances')
plt.show()

# Apply K-Means with the optimal number of clusters
optimal_clusters = 3  # Replace with the optimal number found from the Elbow Method
kmeans = KMeans(n_clusters=optimal_clusters, random_state=42)
data['Cluster'] = kmeans.fit_predict(features)

# Visualize the clusters with a custom color palette
plt.figure(figsize=(12, 8))
custom_palette = ['purple', 'teal', 'blue']  # Replacing yellow with blue
sns.scatterplot(x='Cost of Living Index', y='Local Purchasing Power Index', hue='Cluster', data=data, palette=custom_palette)
plt.title('Clusters of Countries Based on Economic Indices')
plt.show()
# Define the features for clustering
features = data[['Cost of Living Index', 'Rent Index', 'Cost of Living Plus Rent Index', 'Groceries Index', 'Restaurant Price Index', 'Local Purchasing Power Index']]

# Apply K-Means with the optimal number of clusters (based on previous Elbow method, assume 3)
optimal_clusters = 3
kmeans = KMeans(n_clusters=optimal_clusters, random_state=42)
data['Cluster'] = kmeans.fit_predict(features)

# Group countries by their assigned cluster
clustered_countries = data[['Country', 'Cluster']].groupby('Cluster')['Country'].apply(list)

# Display the countries in each cluster
for cluster, countries in clustered_countries.items():
    print(f"Cluster {cluster}: {', '.join(countries)}")

Cluster 0: Barbados, Italy, Cyprus, Uruguay, Jamaica, Malta, Trinidad And Tobago, Costa Rica, Greece, Estonia, Slovenia, Latvia, Spain, Lithuania, Slovakia, Czech Republic, Panama, Croatia, Taiwan, Portugal, Hungary, Poland, Montenegro, Bulgaria, Romania, Fiji, South Africa, China, Malaysia, India
Cluster 1: Cuba, Albania, Lebanon, Palestine, Jordan, Armenia, Mexico, El Salvador, Chile, Guatemala, Venezuela, Dominican Republic, Serbia, Turkey, Cambodia, Cameroon, Zimbabwe, Mauritius, Bosnia And Herzegovina, Sri Lanka, Thailand, Moldova, Georgia, North Macedonia, Ecuador, Kazakhstan, Nigeria, Azerbaijan, Philippines, Russia, Ghana, Brazil, Kenya, Botswana, Peru, Morocco, Kosovo (Disputed Territory), Argentina, Iraq, Uganda, Algeria, Colombia, Vietnam, Tunisia, Bolivia, Kyrgyzstan, Indonesia, Iran, Uzbekistan, Belarus, Ukraine, Nepal, Paraguay, Madagascar, Syria, Tanzania, Bangladesh, Egypt, Libya, Pakistan
Cluster 2: Switzerland, Bahamas, Iceland, Singapore, Norway, Denmark, Hong Kong (China), United States, Australia, Austria, Canada, New Zealand, Ireland, France, Puerto Rico, Finland, Netherlands, Israel, Luxembourg, Germany, United Kingdom, Belgium, South Korea, Sweden, United Arab Emirates, Bahrain, Qatar, Japan, Saudi Arabia, Oman, Kuwait

from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

# Standardize the features before applying PCA
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaled_features = scaler.fit_transform(features)

# Apply PCA
pca = PCA(n_components=2)  # We start with 2 components for visualization purposes
principal_components = pca.fit_transform(scaled_features)

# Add the principal components to the dataframe
data['PC1'] = principal_components[:, 0]
data['PC2'] = principal_components[:, 1]

# Visualize the principal components
plt.figure(figsize=(10, 8))
sns.scatterplot(x='PC1', y='PC2', hue='Cluster', data=data, palette='viridis')
plt.title('PCA of Economic Indices')
plt.xlabel(f'Principal Component 1 ({pca.explained_variance_ratio_[0]*100:.2f}% Variance)')
plt.ylabel(f'Principal Component 2 ({pca.explained_variance_ratio_[1]*100:.2f}% Variance)')
plt.show()

# Get the loadings (correlation between original features and the principal components)
loadings = pd.DataFrame(pca.components_.T, columns=['PC1', 'PC2'], index=features.columns)

# Display the loadings
loadings

import matplotlib.pyplot as plt
import numpy as np

# Explained variance
explained_variance = pca.explained_variance_ratio_
cumulative_variance = np.cumsum(explained_variance)

# Scree plot with cumulative variance
plt.figure(figsize=(10, 6))
bars = plt.bar(range(1, len(explained_variance) + 1), explained_variance, alpha=0.7, label='Individual Explained Variance')

# Adding labels to each bar
for bar in bars:
    yval = bar.get_height()
    plt.text(bar.get_x() + bar.get_width()/2, yval + 0.01, f'{yval:.2%}', ha='center', va='bottom')

plt.plot(range(1, len(cumulative_variance) + 1), cumulative_variance, marker='o', color='red', label='Cumulative Explained Variance')

plt.title('Scree Plot with Cumulative Variance')
plt.xlabel('Principal Component')
plt.ylabel('Explained Variance Ratio')
plt.legend(loc='best')
plt.grid(True)
plt.show()

import statsmodels.api as sm

# Define the predictor (Cost of Living Index) and target (Local Purchasing Power Index)
X = data['Cost of Living Index']
y = data['Local Purchasing Power Index']

# Add a constant to the predictor (required for statsmodels)
X = sm.add_constant(X)

# Fit the simple linear regression model
model = sm.OLS(y, X).fit()

# Print the model summary
print(model.summary())

                                 OLS Regression Results                                 
========================================================================================
Dep. Variable:     Local Purchasing Power Index   R-squared:                       0.480
Model:                                      OLS   Adj. R-squared:                  0.475
Method:                           Least Squares   F-statistic:                     109.8
Date:                          Wed, 21 Aug 2024   Prob (F-statistic):           1.35e-18
Time:                                  20:59:00   Log-Likelihood:                -576.69
No. Observations:                           121   AIC:                             1157.
Df Residuals:                               119   BIC:                             1163.
Df Model:                                     1                                         
Covariance Type:                      nonrobust                                         
========================================================================================
                           coef    std err          t      P>|t|      [0.025      0.975]
----------------------------------------------------------------------------------------
const                   -8.8371      7.522     -1.175      0.242     -23.732       6.058
Cost of Living Index     1.6974      0.162     10.477      0.000       1.377       2.018
==============================================================================
Omnibus:                       17.095   Durbin-Watson:                   1.678
Prob(Omnibus):                  0.000   Jarque-Bera (JB):               32.162
Skew:                           0.594   Prob(JB):                     1.04e-07
Kurtosis:                       5.229   Cond. No.                         134.
==============================================================================

Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.

# Define the predictors (all economic indices) and target (Local Purchasing Power Index)
X = data[['Cost of Living Index', 'Rent Index', 'Cost of Living Plus Rent Index', 'Groceries Index', 'Restaurant Price Index']]
y = data['Local Purchasing Power Index']

# Add a constant to the predictors
X = sm.add_constant(X)

# Fit the multiple linear regression model
model = sm.OLS(y, X).fit()

# Print the model summary
print(model.summary())

                                 OLS Regression Results                                 
========================================================================================
Dep. Variable:     Local Purchasing Power Index   R-squared:                       0.527
Model:                                      OLS   Adj. R-squared:                  0.506
Method:                           Least Squares   F-statistic:                     25.60
Date:                          Wed, 21 Aug 2024   Prob (F-statistic):           2.76e-17
Time:                                  21:02:26   Log-Likelihood:                -570.97
No. Observations:                           121   AIC:                             1154.
Df Residuals:                               115   BIC:                             1171.
Df Model:                                     5                                         
Covariance Type:                      nonrobust                                         
==================================================================================================
                                     coef    std err          t      P>|t|      [0.025      0.975]
--------------------------------------------------------------------------------------------------
const                              6.5503     10.060      0.651      0.516     -13.377      26.478
Cost of Living Index              12.3589     36.920      0.335      0.738     -60.772      85.490
Rent Index                        11.8705     34.217      0.347      0.729     -55.908      79.649
Cost of Living Plus Rent Index   -22.1653     71.260     -0.311      0.756    -163.318     118.987
Groceries Index                   -0.2428      0.638     -0.380      0.704      -1.507       1.022
Restaurant Price Index             0.3652      0.519      0.704      0.483      -0.663       1.393
==============================================================================
Omnibus:                       12.452   Durbin-Watson:                   1.655
Prob(Omnibus):                  0.002   Jarque-Bera (JB):               16.803
Skew:                           0.553   Prob(JB):                     0.000224
Kurtosis:                       4.453   Cond. No.                     2.98e+03
==============================================================================

Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
[2] The condition number is large, 2.98e+03. This might indicate that there are
strong multicollinearity or other numerical problems.

import seaborn as sns

# Residual plot to check for linearity and homoscedasticity
sns.residplot(x=model.fittedvalues, y=model.resid, lowess=True, line_kws={'color': 'red'})
plt.title('Residual Plot')
plt.xlabel('Fitted Values')
plt.ylabel('Residuals')
plt.show()

# Variance Inflation Factor (VIF) to check for multicollinearity
from statsmodels.stats.outliers_influence import variance_inflation_factor

vif_data = pd.DataFrame()
vif_data["feature"] = X.columns
vif_data["VIF"] = [variance_inflation_factor(X.values, i) for i in range(len(X.columns))]
print(vif_data)

                          feature            VIF
0                           const      15.838952
1            Cost of Living Index   55159.466869
2                      Rent Index   23666.278859
3  Cost of Living Plus Rent Index  138648.894538
4                 Groceries Index      18.392194
5          Restaurant Price Index      13.929992

import geopandas as gpd
import folium
import branca.colormap as cm

# Load the world shapefile from the downloaded data
world = gpd.read_file(r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\ne_110m_admin_0_countries.shp')

# Merge the world map with the economic data based on country names
world = world.merge(data, how="left", left_on="NAME", right_on="Country")

# Create a folium map centered around a particular location
m = folium.Map(location=[0, 0], zoom_start=2)

# Define a color scale using branca colormap
colormap = cm.linear.PuBuGn_09.scale(world['Cost of Living Index'].min(), world['Cost of Living Index'].max())
colormap = colormap.to_step(index=[20, 40, 60, 80, 100])

# Add a choropleth layer to the map for the Cost of Living Index
folium.Choropleth(
    geo_data=world,
    name="choropleth",
    data=world,
    columns=["NAME", "Cost of Living Index"],
    key_on="feature.properties.NAME",
    fill_color='PuBuGn',
    fill_opacity=0.7,
    line_opacity=0.2,
    nan_fill_color="black",  # Clearly indicate countries with missing data
    nan_fill_opacity=0.4
).add_to(m)

# Customize the legend placement and add description
colormap.caption = 'Cost of Living Index (relative to NYC = 100)'
colormap.add_to(m)

# Adjust the position of the legend to the left or right
colormap.add_to(m)

# Add additional layers as needed for other indices
folium.LayerControl().add_to(m)

# Display the map
m.save("cost_of_living_map_with_side_legend.html")
m

import geopandas as gpd
import pandas as pd
import folium
from folium.plugins import HeatMap
import branca.colormap as cm

# Step 1: Load the Shapefile and CSV File
shapefile_path = r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\ne_110m_admin_0_countries.shp'
csv_path = r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\Cost_of_Living_Index_by_Country_2024.csv'

world = gpd.read_file(shapefile_path)
data = pd.read_csv(csv_path)

# Step 2: Map and Align Country Names (If Necessary)
country_mapping = {
    "United States": "United States of America",
    "South Korea": "Republic of Korea",
    "Vietnam": "Viet Nam",
    # Add more mappings as necessary
}

data['Country'] = data['Country'].replace(country_mapping)

# Step 3: Merge the GeoDataFrame and DataFrame
merged = world.merge(data, how="left", left_on="SOVEREIGNT", right_on="Country")

# Step 4: Drop rows with NaN values in the 'Local Purchasing Power Index' column
merged = merged.dropna(subset=['Local Purchasing Power Index'])

# Step 5: Create the Heatmap
# Generate the heat data
heat_data = [
    [row['geometry'].centroid.y, row['geometry'].centroid.x, row['Local Purchasing Power Index']] 
    for index, row in merged.iterrows()
]

# Initialize the map
m = folium.Map(location=[20, 0], zoom_start=2)

# Add the heatmap
HeatMap(heat_data, radius=15).add_to(m)

# Create a color scale legend using branca
colormap = cm.LinearColormap(
    colors=['blue', 'green', 'yellow', 'red'],  # Color gradient
    vmin=merged['Local Purchasing Power Index'].min(), 
    vmax=merged['Local Purchasing Power Index'].max(),
    caption="Local Purchasing Power Index"
)

# Add the color scale to the map
colormap.add_to(m)

# Step 6: Save and Display the Map
m.save("purchasing_power_heatmap_with_legend.html")
m

import geopandas as gpd
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# Load your CSV and shapefile as previously done
shapefile_path = '/mnt/data/ne_110m_admin_0_countries.shp'
csv_path = '/mnt/data/Cost_of_Living_Index_by_Country_2024.csv'

world = gpd.read_file(r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\ne_110m_admin_0_countries.shp')
data = pd.read_csv(r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\Cost_of_Living_Index_by_Country_2024.csv')

# Region mapping (as previously defined)
region_mapping = {
    "United States of America": "North America", "Canada": "North America", "Mexico": "North America",
    "Brazil": "South America", "Argentina": "South America", "Chile": "South America",
    "Germany": "Europe", "France": "Europe", "United Kingdom": "Europe", "Italy": "Europe",
    "Spain": "Europe", "China": "Asia", "India": "Asia", "Japan": "Asia", "South Korea": "Asia",
    "Russia": "Europe", "Australia": "Oceania", "New Zealand": "Oceania", "South Africa": "Africa",
    "Nigeria": "Africa", "Egypt": "Africa", "Morocco": "Africa", "Saudi Arabia": "Middle East",
    "United Arab Emirates": "Middle East", "Israel": "Middle East", "Turkey": "Middle East",
    "Indonesia": "Asia", "Vietnam": "Asia", "Philippines": "Asia", "Thailand": "Asia",
    "Malaysia": "Asia", "Singapore": "Asia", "Switzerland": "Europe", "Austria": "Europe",
    "Netherlands": "Europe", "Belgium": "Europe", "Sweden": "Europe", "Norway": "Europe",
    "Denmark": "Europe", "Finland": "Europe", "Ireland": "Europe", "Poland": "Europe",
    "Czech Republic": "Europe", "Hungary": "Europe", "Portugal": "Europe", "Greece": "Europe",
    "Turkey": "Middle East", "Ukraine": "Europe", "Belarus": "Europe", "Kazakhstan": "Asia",
    "Uzbekistan": "Asia", "Kyrgyzstan": "Asia", "Armenia": "Asia", "Azerbaijan": "Asia",
    "Georgia": "Europe", "Sri Lanka": "Asia", "Bangladesh": "Asia", "Pakistan": "Asia",
    "Nepal": "Asia", "Bhutan": "Asia", "Maldives": "Asia", "Myanmar": "Asia", "Cambodia": "Asia",
    "Laos": "Asia", "Brunei": "Asia", "Papua New Guinea": "Oceania", "Fiji": "Oceania",
    "New Caledonia": "Oceania", "Solomon Islands": "Oceania", "Vanuatu": "Oceania", "Tonga": "Oceania",
    "Samoa": "Oceania", "Iceland": "Europe", "Greenland": "North America", "Jamaica": "North America",
    "Cuba": "North America", "Dominican Republic": "North America", "Haiti": "North America",
    "Trinidad And Tobago": "North America", "Panama": "North America", "Costa Rica": "North America",
    "El Salvador": "North America", "Honduras": "North America", "Guatemala": "North America",
    "Nicaragua": "North America", "Venezuela": "South America", "Colombia": "South America",
    "Peru": "South America", "Ecuador": "South America", "Bolivia": "South America",
    "Paraguay": "South America", "Uruguay": "South America", "Brazil": "South America",
    "Suriname": "South America", "Guyana": "South America", "French Guiana": "South America",
    "Libya": "Africa", "Tunisia": "Africa", "Algeria": "Africa", "Morocco": "Africa", "Mali": "Africa",
    "Niger": "Africa", "Chad": "Africa", "Sudan": "Africa", "South Sudan": "Africa", "Ethiopia": "Africa",
    "Somalia": "Africa", "Kenya": "Africa", "Uganda": "Africa", "Tanzania": "Africa", "Rwanda": "Africa",
    "Burundi": "Africa", "Democratic Republic of the Congo": "Africa", "Republic of the Congo": "Africa",
    "Angola": "Africa", "Zambia": "Africa", "Malawi": "Africa", "Mozambique": "Africa", "Zimbabwe": "Africa",
    "Botswana": "Africa", "Namibia": "Africa", "South Africa": "Africa", "Lesotho": "Africa",
    "Eswatini": "Africa", "Madagascar": "Africa", "Mauritius": "Africa", "Comoros": "Africa",
    "Seychelles": "Africa", "Djibouti": "Africa", "Eritrea": "Africa", "Saudi Arabia": "Middle East",
    "Yemen": "Middle East", "Oman": "Middle East", "United Arab Emirates": "Middle East", "Qatar": "Middle East",
    "Bahrain": "Middle East", "Kuwait": "Middle East", "Iran": "Middle East", "Iraq": "Middle East",
    "Syria": "Middle East", "Lebanon": "Middle East", "Jordan": "Middle East", "Palestine": "Middle East",
    "Israel": "Middle East", "Afghanistan": "Asia", "Tajikistan": "Asia", "Turkmenistan": "Asia",
    "Uzbekistan": "Asia", "Kyrgyzstan": "Asia", "Kazakhstan": "Asia", "Mongolia": "Asia"
}


# Add Region column based on mapping
data['Region'] = data['Country'].map(region_mapping)

# Group by region and calculate mean of each index
region_grouped = data.groupby('Region').agg({
    'Local Purchasing Power Index': 'mean',
    'Cost of Living Index': 'mean',
    'Rent Index': 'mean',
    'Groceries Index': 'mean',
    'Restaurant Price Index': 'mean'
}).reset_index()


# Assuming 'region_grouped' is your DataFrame with the regional averages
plt.figure(figsize=(12, 8))

# Use lighter colors for the bars
colors = ['lightcoral', 'lightseagreen', 'lightskyblue', 'lightpink', 'lightgreen', 'lightblue', 'lightgoldenrodyellow']

# Plot 1: Bar Plot of Average Local Purchasing Power Index by Region
plt.bar(region_grouped['Region'], region_grouped['Local Purchasing Power Index'], color=colors)
plt.title('Average Local Purchasing Power Index by Region')
plt.xlabel('Region')
plt.ylabel('Local Purchasing Power Index')
plt.xticks(rotation=45, ha='right')
plt.tight_layout()
plt.show()  # Render this plot

# Plot 2: Facet Grid of Cost of Living Index vs. Local Purchasing Power Index by Region
plt.figure(figsize=(12, 8))  # Ensure a new figure is created
g = sns.FacetGrid(data, col="Region", col_wrap=4, height=4, aspect=1.2)
g.map(sns.scatterplot, "Cost of Living Index", "Local Purchasing Power Index", alpha=0.7)
g.fig.suptitle("Cost of Living Index vs. Local Purchasing Power Index by Region", fontsize=16)
g.fig.subplots_adjust(top=0.9)
plt.show()  # Render this plot

# Plot 3: Pair Plot across the Regions
plt.figure(figsize=(12, 8))  # Ensure a new figure is created
sns.pairplot(data, hue="Region", diag_kind="kde", height=2.5)
plt.suptitle("Pair Plot of Economic Indices by Region", y=1.02, fontsize=16)
plt.show()  # Render this plot

<Figure size 1200x800 with 0 Axes>

<Figure size 1200x800 with 0 Axes>

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# Dropping any rows with missing values
data_ml = data.dropna()

# Convert categorical variables to dummy variables (one-hot encoding)
data_ml = pd.get_dummies(data_ml, drop_first=True)

# Separate features (X) and target (y)
X = data_ml.drop(columns=['Local Purchasing Power Index'])  # Replace with your actual target column
y = data_ml['Local Purchasing Power Index']

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Standardize the features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.metrics import mean_squared_error, r2_score

# Initialize the models
models = {
    'Linear Regression': LinearRegression(),
    'Ridge Regression': Ridge(),
    'Lasso Regression': Lasso(),
    'Random Forest': RandomForestRegressor(n_estimators=100, random_state=42),
    'Gradient Boosting': GradientBoostingRegressor(n_estimators=100, random_state=42)
}

# Dictionary to store the results
results = {}

# Train and evaluate each model
for name, model in models.items():
    # Train the model
    model.fit(X_train_scaled, y_train)
    
    # Make predictions
    y_pred = model.predict(X_test_scaled)
    
    # Evaluate the model
    mse = mean_squared_error(y_test, y_pred)
    r2 = r2_score(y_test, y_pred)
    
    # Store the results
    results[name] = {'MSE': mse, 'R-squared': r2}

# Display the results
for model_name, metrics in results.items():
    print(f"{model_name} - MSE: {metrics['MSE']:.4f}, R-squared: {metrics['R-squared']:.4f}")

Linear Regression - MSE: 575.9529, R-squared: 0.6028
Ridge Regression - MSE: 356.1119, R-squared: 0.7544
Lasso Regression - MSE: 516.5958, R-squared: 0.6437
Random Forest - MSE: 393.4383, R-squared: 0.7287
Gradient Boosting - MSE: 517.3378, R-squared: 0.6432

from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import Ridge

# Define the parameter grid for Ridge Regression
param_grid = {
    'alpha': [0.1, 1.0, 10.0, 100.0],  # Regularization strength
    'solver': ['auto', 'svd', 'cholesky', 'lsqr', 'sag']  # Solvers for Ridge regression
}

# Initialize GridSearchCV
grid_search = GridSearchCV(estimator=Ridge(), param_grid=param_grid, cv=5, scoring='r2', n_jobs=-1)

# Fit the model with GridSearchCV
grid_search.fit(X_train_scaled, y_train)

# Best parameters and score
best_params = grid_search.best_params_
best_score = grid_search.best_score_

# Evaluate the best model on the test set
best_model = grid_search.best_estimator_
y_pred_best = best_model.predict(X_test_scaled)
mse_best = mean_squared_error(y_test, y_pred_best)
r2_best = r2_score(y_test, y_pred_best)

print(f"Best Parameters: {best_params}")
print(f"Best Cross-Validation Score: {best_score:.4f}")
print(f"Test Set MSE: {mse_best:.4f}")
print(f"Test Set R-squared: {r2_best:.4f}")

Best Parameters: {'alpha': 0.1, 'solver': 'auto'}
Best Cross-Validation Score: 0.5028
Test Set MSE: 356.6717
Test Set R-squared: 0.7540

import pandas as pd
import matplotlib.pyplot as plt
import pandas as pd

# Get the coefficients of the Ridge Regression model
coefficients = best_model.coef_

# Create a DataFrame to pair feature names with their coefficients
coef_df = pd.DataFrame({
    'Feature': X.columns,
    'Coefficient': coefficients
})

# Sort the DataFrame by the absolute value of the coefficients
coef_df['Abs_Coefficient'] = coef_df['Coefficient'].abs()
coef_df = coef_df.sort_values(by='Abs_Coefficient', ascending=False)

# Display the top features
print(coef_df[['Feature', 'Coefficient']])


# Assuming 'coef_df' is your DataFrame with feature names and coefficients

# Sort the DataFrame by the absolute value of coefficients
coef_df['abs_coef'] = coef_df['Coefficient'].abs()
coef_df = coef_df.sort_values(by='abs_coef', ascending=False)

# Select top 10 and bottom 10 features
top_features = pd.concat([coef_df.head(10), coef_df.tail(10)])

                 Feature  Coefficient
51        Country_Kuwait     9.894481
65          Country_Oman     8.096656
76  Country_Saudi Arabia     7.370909
78  Country_South Africa     6.390041
39         Country_India     6.088766
..                   ...          ...
73      Country_Portugal     0.000000
74         Country_Qatar     0.000000
23    Country_Costa Rica     0.000000
77     Country_Singapore     0.000000
72        Country_Poland     0.000000

[105 rows x 2 columns]

# Assuming 'coef_df' is your DataFrame with feature names and coefficients

# Sort the DataFrame by the absolute value of coefficients
coef_df['abs_coef'] = coef_df['Coefficient'].abs()
coef_df = coef_df.sort_values(by='abs_coef', ascending=False)

# Select top 10 and bottom 10 features
top_features = pd.concat([coef_df.head(10), coef_df.tail(10)])

# Create a compact figure with a grid style
plt.figure(figsize=(10, 6))
sns.set(style="whitegrid")

# Plot the coefficients with annotations
barplot = sns.barplot(x="Coefficient", y="Feature", data=top_features, palette="coolwarm")

# Add annotations to each bar
for index, value in enumerate(top_features['Coefficient']):
    plt.text(value, index, f'{value:.2f}', color='black', va="center", ha="left" if value < 0 else "right")

# Add titles and labels
plt.title('Top 10 and Bottom 10 Feature Importances', fontsize=16)
plt.xlabel('Coefficient Value', fontsize=14)
plt.ylabel('Features', fontsize=14)

# Adjust the plot to ensure everything fits without overlapping
plt.tight_layout()

plt.show()

# Import necessary libraries
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from xgboost import XGBRegressor
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.linear_model import Ridge

# Assuming X_train, X_test, y_train, y_test are already defined from previous steps

# Define the models and hyperparameters to tune
models_and_parameters = {
    'Ridge': {
        'model': Ridge(),
        'params': {
            'alpha': [0.1, 1, 10, 100],
            'solver': ['auto', 'svd', 'cholesky', 'lsqr', 'sag']
        }
    },
    'RandomForest': {
        'model': RandomForestRegressor(),
        'params': {
            'n_estimators': [50, 100, 200],
            'max_features': ['auto', 'sqrt', 'log2'],
            'max_depth': [None, 10, 20, 30],
            'min_samples_split': [2, 5, 10],
            'min_samples_leaf': [1, 2, 4]
        }
    },
    'GradientBoosting': {
        'model': GradientBoostingRegressor(),
        'params': {
            'n_estimators': [50, 100, 200],
            'learning_rate': [0.01, 0.1, 0.2],
            'max_depth': [3, 5, 7],
            'subsample': [0.8, 0.9, 1.0],
            'max_features': ['auto', 'sqrt', 'log2']
        }
    },
    'XGBoost': {
        'model': XGBRegressor(),
        'params': {
            'n_estimators': [50, 100, 200],
            'learning_rate': [0.01, 0.1, 0.2],
            'max_depth': [3, 5, 7],
            'subsample': [0.8, 0.9, 1.0],
            'colsample_bytree': [0.8, 0.9, 1.0]
        }
    }
}

# Function to perform grid search and print the best results
def refine_model(model_name, model, params, X_train, y_train, X_test, y_test):
    grid_search = GridSearchCV(model, params, cv=5, scoring='neg_mean_squared_error', n_jobs=-1, verbose=1)
    grid_search.fit(X_train, y_train)
    
    # Best parameters and best score from cross-validation
    print(f"Best parameters for {model_name}: {grid_search.best_params_}")
    print(f"Best cross-validation score for {model_name}: {-grid_search.best_score_}")
    
    # Test the best model on the test set
    best_model = grid_search.best_estimator_
    y_pred = best_model.predict(X_test)
    
    mse = mean_squared_error(y_test, y_pred)
    r2 = r2_score(y_test, y_pred)
    
    print(f"{model_name} Test Set MSE: {mse}")
    print(f"{model_name} Test Set R-squared: {r2}\n")
    
    return best_model

# Loop over the models and parameters
best_models = {}
for model_name, mp in models_and_parameters.items():
    best_model = refine_model(model_name, mp['model'], mp['params'], X_train, y_train, X_test, y_test)
    best_models[model_name] = best_model

# After tuning, you can compare the best models and choose the most appropriate one

Fitting 5 folds for each of 20 candidates, totalling 100 fits
Best parameters for Ridge: {'alpha': 1, 'solver': 'svd'}
Best cross-validation score for Ridge: 975.7798253988352
Ridge Test Set MSE: 564.3317926356939
Ridge Test Set R-squared: 0.6108066943127572

Fitting 5 folds for each of 324 candidates, totalling 1620 fits
Best parameters for RandomForest: {'max_depth': 30, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 5, 'n_estimators': 50}
Best cross-validation score for RandomForest: 881.2965750042989
RandomForest Test Set MSE: 379.33780870519945
RandomForest Test Set R-squared: 0.7383884132194583

Fitting 5 folds for each of 243 candidates, totalling 1215 fits
Best parameters for GradientBoosting: {'learning_rate': 0.2, 'max_depth': 7, 'max_features': 'log2', 'n_estimators': 50, 'subsample': 0.9}
Best cross-validation score for GradientBoosting: 788.6545787014338
GradientBoosting Test Set MSE: 505.2196060733247
GradientBoosting Test Set R-squared: 0.6515736112131154

Fitting 5 folds for each of 243 candidates, totalling 1215 fits
Best parameters for XGBoost: {'colsample_bytree': 0.8, 'learning_rate': 0.2, 'max_depth': 3, 'n_estimators': 100, 'subsample': 0.8}
Best cross-validation score for XGBoost: 975.8462322065692
XGBoost Test Set MSE: 589.133550443729
XGBoost Test Set R-squared: 0.593702079201351

## Shapiro-Wilk Test: This tests if the data is normally distributed.

from scipy.stats import shapiro

# Example: Check normality for Cost of Living Index in a specific region
stat, p_value = shapiro(data['Cost of Living Index'])
print('Shapiro-Wilk Test: Statistics=%.3f, p=%.3f' % (stat, p_value))

# Interpretation: If p > 0.05, the data is normally distributed.

Shapiro-Wilk Test: Statistics=0.935, p=0.000

## Q-Q Plot: Visualize if the data follows a normal distribution.

import statsmodels.api as sm
import matplotlib.pyplot as plt

# Q-Q plot for Cost of Living Index
sm.qqplot(data['Cost of Living Index'], line ='45')
plt.show()

## Levene's Test: This tests if the variances are equal across groups.

from scipy.stats import levene

# Map countries to regions in the DataFrame
data['Region'] = data['Country'].map(region_mapping)

# Ensure that the data has no NaN values after mapping
data.dropna(subset=['Region'], inplace=True)

# Levene's Test: This tests if the variances are equal across groups.
stat, p_value = levene(data['Cost of Living Index'][data['Region'] == 'Europe'],
                       data['Cost of Living Index'][data['Region'] == 'Asia'],
                       data['Cost of Living Index'][data['Region'] == 'Africa'])

print('Levene’s Test: Statistics=%.3f, p=%.3f' % (stat, p_value))

# Interpretation: If p > 0.05, variances are equal.

Levene’s Test: Statistics=3.867, p=0.027

import seaborn as sns

# Scatter plot to check linearity between Cost of Living Index and Local Purchasing Power Index
sns.scatterplot(x='Cost of Living Index', y='Local Purchasing Power Index', data=data)
plt.show()

import pandas as pd
import scipy.stats as stats

# Load the dataset
data = pd.read_csv(r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\Cost_of_Living_Index_by_Country_2024.csv')

# Define a threshold for high-income and low-income countries
# For example, let's assume the threshold for high income is a Local Purchasing Power Index above 60
income_threshold = 60

# Create the Income_Group column
data['Income_Group'] = data['Local Purchasing Power Index'].apply(lambda x: 1 if x > income_threshold else 0)

# Check the first few rows to confirm the new column
print(data[['Country', 'Local Purchasing Power Index', 'Income_Group']].head())

# Save the updated dataframe if needed
# data.to_csv('Cost_of_Living_Index_with_Income_Group.csv', index=False)

# Mann-Whitney U Test: Economic indices between high-income and low-income countries
high_income = data[data['Income_Group'] == 1]['Cost of Living Index']
low_income = data[data['Income_Group'] == 0]['Cost of Living Index']

# Perform the Mann-Whitney U Test
mannwhitney_result = stats.mannwhitneyu(high_income, low_income)

# Display the results
print(f"Mann-Whitney U Test: Statistics={mannwhitney_result.statistic}, p-value={mannwhitney_result.pvalue}")

# Spearman's Rank Correlation between Cost of Living Index and Local Purchasing Power Index
spearman_result = stats.spearmanr(data['Cost of Living Index'], data['Local Purchasing Power Index'])

# Display the results
print(f"Spearman's Rank Correlation: Correlation={spearman_result.correlation}, p-value={spearman_result.pvalue}")

     Country  Local Purchasing Power Index  Income_Group
0    Albania                          39.9             0
1    Algeria                          29.9             0
2  Argentina                          41.5             0
3    Armenia                          38.5             0
4  Australia                         127.4             1
Mann-Whitney U Test: Statistics=3100.5, p-value=8.021521482986537e-12
Spearman's Rank Correlation: Correlation=0.7053094199200701, p-value=1.701774651887184e-19

# Import necessary libraries
import pandas as pd

# Load the datasets
cost_of_living_df = pd.read_csv(r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\Cost_of_Living_Index_by_Country_2024.csv')
gdp_df = pd.read_csv(r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\Economic Analysis data\GDP per capita (current US$\GDP 2023.csv')
gni_df = pd.read_csv(r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\Economic Analysis data\GNI per capita, PPP (current international $)\GNI 2023.csv')

# Display the first few rows of each dataframe to inspect the data
print(cost_of_living_df.head())
print(gdp_df.head())
print(gni_df.head())

# Standardize country names if necessary
# (Assuming the 'Country' column exists in all datasets)
# For simplicity, let's assume the column name for countries is 'Country' in all datasets
cost_of_living_df['Country'] = cost_of_living_df['Country'].str.strip()
gdp_df['Country'] = gdp_df['Country'].str.strip()
gni_df['Country'] = gni_df['Country'].str.strip()

# Check for missing values in key columns
print(cost_of_living_df.isnull().sum())
print(gdp_df.isnull().sum())
print(gni_df.isnull().sum())

# Merge the datasets on 'Country'
merged_df = cost_of_living_df.merge(gdp_df, on='Country', how='inner')
merged_df = merged_df.merge(gni_df, on='Country', how='inner')

# Display the first few rows of the merged dataframe
print(merged_df.head())

# Check for any remaining missing values after merging
print(merged_df.isnull().sum())

   Rank    Country  Cost of Living Index  Rent Index  \
0    53    Albania                  42.1        10.6   
1   101    Algeria                  28.9         3.8   
2    98  Argentina                  29.4         7.6   
3    58    Armenia                  41.0        19.0   
4    10  Australia                  70.2        33.4   

   Cost of Living Plus Rent Index  Groceries Index  Restaurant Price Index  \
0                            27.0             42.0                    35.7   
1                            16.9             36.8                    14.0   
2                            18.9             29.7                    24.8   
3                            30.5             36.0                    38.9   
4                            52.5             77.3                    62.5   

   Local Purchasing Power Index  
0                          39.9  
1                          29.9  
2                          41.5  
3                          38.5  
4                         127.4  
                       Country         2023
0                        Aruba          NaN
1  Africa Eastern and Southern  1672.505957
2                  Afghanistan          NaN
3   Africa Western and Central  1584.333285
4                       Angola  2309.521620
                       Country         2023
0                        Aruba          NaN
1  Africa Eastern and Southern  4355.819919
2                  Afghanistan          NaN
3   Africa Western and Central  5239.787316
4                       Angola  7310.000000
Rank                              0
Country                           0
Cost of Living Index              0
Rent Index                        0
Cost of Living Plus Rent Index    0
Groceries Index                   0
Restaurant Price Index            0
Local Purchasing Power Index      0
dtype: int64
Country     0
2023       31
dtype: int64
Country     0
2023       37
dtype: int64
   Rank    Country  Cost of Living Index  Rent Index  \
0    53    Albania                  42.1        10.6   
1   101    Algeria                  28.9         3.8   
2    98  Argentina                  29.4         7.6   
3    58    Armenia                  41.0        19.0   
4    10  Australia                  70.2        33.4   

   Cost of Living Plus Rent Index  Groceries Index  Restaurant Price Index  \
0                            27.0             42.0                    35.7   
1                            16.9             36.8                    14.0   
2                            18.9             29.7                    24.8   
3                            30.5             36.0                    38.9   
4                            52.5             77.3                    62.5   

   Local Purchasing Power Index        2023_x   2023_y  
0                          39.9   8367.775731  21110.0  
1                          29.9   5260.206250  16790.0  
2                          41.5  13730.514710  28710.0  
3                          38.5   8715.765336  22440.0  
4                         127.4  64711.765600  66260.0  
Rank                              0
Country                           0
Cost of Living Index              0
Rent Index                        0
Cost of Living Plus Rent Index    0
Groceries Index                   0
Restaurant Price Index            0
Local Purchasing Power Index      0
2023_x                            2
2023_y                            3
dtype: int64

# Rename columns for clarity
merged_df.rename(columns={'2023_x': 'GDP_2023', '2023_y': 'GNI_2023'}, inplace=True)

# Handle missing data by dropping rows with NaNs
# Alternatively, you could use imputation methods here if you want to retain more data
merged_df.dropna(subset=['GDP_2023', 'GNI_2023'], inplace=True)

# Calculate Economic Efficiency Metrics
merged_df['Cost_of_Living_Efficiency_Ratio'] = merged_df['Cost of Living Index'] / merged_df['Local Purchasing Power Index']
merged_df['Income_Efficiency_Ratio'] = merged_df['GDP_2023'] / merged_df['GNI_2023']

# Display the updated DataFrame with new metrics
print(merged_df.head())

# Check for any remaining missing values in key columns
print(merged_df.isnull().sum())

   Rank    Country  Cost of Living Index  Rent Index  \
0    53    Albania                  42.1        10.6   
1   101    Algeria                  28.9         3.8   
2    98  Argentina                  29.4         7.6   
3    58    Armenia                  41.0        19.0   
4    10  Australia                  70.2        33.4   

   Cost of Living Plus Rent Index  Groceries Index  Restaurant Price Index  \
0                            27.0             42.0                    35.7   
1                            16.9             36.8                    14.0   
2                            18.9             29.7                    24.8   
3                            30.5             36.0                    38.9   
4                            52.5             77.3                    62.5   

   Local Purchasing Power Index      GDP_2023  GNI_2023  \
0                          39.9   8367.775731   21110.0   
1                          29.9   5260.206250   16790.0   
2                          41.5  13730.514710   28710.0   
3                          38.5   8715.765336   22440.0   
4                         127.4  64711.765600   66260.0   

   Cost_of_Living_Efficiency_Ratio  Income_Efficiency_Ratio  
0                         1.055138                 0.396389  
1                         0.966555                 0.313294  
2                         0.708434                 0.478249  
3                         1.064935                 0.388403  
4                         0.551020                 0.976634  
Rank                               0
Country                            0
Cost of Living Index               0
Rent Index                         0
Cost of Living Plus Rent Index     0
Groceries Index                    0
Restaurant Price Index             0
Local Purchasing Power Index       0
GDP_2023                           0
GNI_2023                           0
Cost_of_Living_Efficiency_Ratio    0
Income_Efficiency_Ratio            0
dtype: int64

# Calculate descriptive statistics for Cost_of_Living_Efficiency_Ratio and Income_Efficiency_Ratio
descriptive_stats = merged_df[['Cost_of_Living_Efficiency_Ratio', 'Income_Efficiency_Ratio']].describe()
print(descriptive_stats)

       Cost_of_Living_Efficiency_Ratio  Income_Efficiency_Ratio
count                       100.000000               100.000000
mean                          0.815309                 0.537753
std                           0.484284                 0.242261
min                           0.256659                 0.230282
25%                           0.553827                 0.352719
50%                           0.684850                 0.472637
75%                           0.883646                 0.657206
max                           3.552381                 1.302258

# Top 5 most efficient countries based on Cost_of_Living_Efficiency_Ratio
top_5_cost_efficiency = merged_df[['Country', 'Cost_of_Living_Efficiency_Ratio']].sort_values(by='Cost_of_Living_Efficiency_Ratio', ascending=True).head(5)
print("Top 5 Most Cost Efficient Countries:\n", top_5_cost_efficiency)

# Bottom 5 least efficient countries based on Cost_of_Living_Efficiency_Ratio
bottom_5_cost_efficiency = merged_df[['Country', 'Cost_of_Living_Efficiency_Ratio']].sort_values(by='Cost_of_Living_Efficiency_Ratio', ascending=False).head(5)
print("Bottom 5 Least Cost Efficient Countries:\n", bottom_5_cost_efficiency)

# Top 5 most efficient countries based on Income_Efficiency_Ratio
top_5_income_efficiency = merged_df[['Country', 'Income_Efficiency_Ratio']].sort_values(by='Income_Efficiency_Ratio', ascending=True).head(5)
print("Top 5 Most Income Efficient Countries:\n", top_5_income_efficiency)

# Bottom 5 least efficient countries based on Income_Efficiency_Ratio
bottom_5_income_efficiency = merged_df[['Country', 'Income_Efficiency_Ratio']].sort_values(by='Income_Efficiency_Ratio', ascending=False).head(5)
print("Bottom 5 Least Income Efficient Countries:\n", bottom_5_income_efficiency)

Top 5 Most Cost Efficient Countries:
          Country  Cost_of_Living_Efficiency_Ratio
42         India                         0.256659
54        Kuwait                         0.260198
74          Oman                         0.303290
88  South Africa                         0.335603
85  Saudi Arabia                         0.335804
Bottom 5 Least Cost Efficient Countries:
       Country  Cost_of_Living_Efficiency_Ratio
17   Cameroon                         3.552381
71    Nigeria                         2.854545
90  Sri Lanka                         1.977143
9    Barbados                         1.760920
37      Ghana                         1.679348
Top 5 Most Income Efficient Countries:
         Country  Income_Efficiency_Ratio
75     Pakistan                 0.230282
42        India                 0.247741
68        Nepal                 0.252678
101  Uzbekistan                 0.258932
71      Nigeria                 0.261472
Bottom 5 Least Income Efficient Countries:
         Country  Income_Efficiency_Ratio
59   Luxembourg                 1.302258
9      Barbados                 1.222243
82  Puerto Rico                 1.114855
92  Switzerland                 1.110068
45      Ireland                 1.051038

import matplotlib.pyplot as plt
import seaborn as sns
plt.figure(figsize=(12, 6))

# Histogram for Cost_of_Living_Efficiency_Ratio
plt.subplot(1, 2, 1)
sns.histplot(merged_df['Cost_of_Living_Efficiency_Ratio'], bins=20, kde=True, color='skyblue')
plt.title('Distribution of Cost of Living Efficiency Ratio')
plt.xlabel('Cost of Living Efficiency Ratio')
plt.ylabel('Frequency')

# Histogram for Income_Efficiency_Ratio
plt.subplot(1, 2, 2)
sns.histplot(merged_df['Income_Efficiency_Ratio'], bins=20, kde=True, color='lightgreen')
plt.title('Distribution of Income Efficiency Ratio')
plt.xlabel('Income Efficiency Ratio')
plt.ylabel('Frequency')

plt.tight_layout()
plt.show()

plt.figure(figsize=(12, 6))

# Box plot for Cost_of_Living_Efficiency_Ratio
plt.subplot(1, 2, 1)
sns.boxplot(y=merged_df['Cost_of_Living_Efficiency_Ratio'], color='skyblue')
plt.title('Box Plot of Cost of Living Efficiency Ratio')
plt.xlabel('Cost of Living Efficiency Ratio')

# Box plot for Income_Efficiency_Ratio
plt.subplot(1, 2, 2)
sns.boxplot(y=merged_df['Income_Efficiency_Ratio'], color='lightgreen')
plt.title('Box Plot of Income Efficiency Ratio')
plt.xlabel('Income Efficiency Ratio')

plt.tight_layout()
plt.show()

# Define a mapping of countries to their regions
country_to_region = {
    "United States": "North America", "Canada": "North America", "Mexico": "North America",
    "Brazil": "South America", "Argentina": "South America", "Chile": "South America",
    "Germany": "Europe", "France": "Europe", "United Kingdom": "Europe", "Italy": "Europe",
    "Spain": "Europe", "China": "Asia", "India": "Asia", "Japan": "Asia", "South Korea": "Asia",
    "Russia": "Europe", "Australia": "Oceania", "New Zealand": "Oceania", "South Africa": "Africa",
    "Nigeria": "Africa", "Egypt": "Africa", "Morocco": "Africa", "Saudi Arabia": "Middle East",
    "United Arab Emirates": "Middle East", "Israel": "Middle East", "Turkey": "Middle East",
    "Indonesia": "Asia", "Vietnam": "Asia", "Philippines": "Asia", "Thailand": "Asia",
    "Malaysia": "Asia", "Singapore": "Asia", "Switzerland": "Europe", "Austria": "Europe",
    "Netherlands": "Europe", "Belgium": "Europe", "Sweden": "Europe", "Norway": "Europe",
    "Denmark": "Europe", "Finland": "Europe", "Ireland": "Europe", "Poland": "Europe",
    "Czech Republic": "Europe", "Hungary": "Europe", "Portugal": "Europe", "Greece": "Europe",
    "Ukraine": "Europe", "Belarus": "Europe", "Kazakhstan": "Asia", "Uzbekistan": "Asia",
    "Kyrgyzstan": "Asia", "Armenia": "Asia", "Azerbaijan": "Asia", "Georgia": "Europe",
    "Sri Lanka": "Asia", "Bangladesh": "Asia", "Pakistan": "Asia", "Nepal": "Asia", "Bhutan": "Asia",
    "Maldives": "Asia", "Myanmar": "Asia", "Cambodia": "Asia", "Laos": "Asia", "Brunei": "Asia",
    "Papua New Guinea": "Oceania", "Fiji": "Oceania", "New Caledonia": "Oceania", "Solomon Islands": "Oceania",
    "Vanuatu": "Oceania", "Tonga": "Oceania", "Samoa": "Oceania", "Iceland": "Europe", "Greenland": "North America",
    "Jamaica": "North America", "Cuba": "North America", "Dominican Republic": "North America", "Haiti": "North America",
    "Trinidad And Tobago": "North America", "Panama": "North America", "Costa Rica": "North America", "El Salvador": "North America",
    "Honduras": "North America", "Guatemala": "North America", "Nicaragua": "North America", "Venezuela": "South America",
    "Colombia": "South America", "Peru": "South America", "Ecuador": "South America", "Bolivia": "South America", "Paraguay": "South America",
    "Uruguay": "South America", "Suriname": "South America", "Guyana": "South America", "French Guiana": "South America",
    "Libya": "Africa", "Tunisia": "Africa", "Algeria": "Africa", "Morocco": "Africa", "Mali": "Africa", "Niger": "Africa",
    "Chad": "Africa", "Sudan": "Africa", "South Sudan": "Africa", "Ethiopia": "Africa", "Somalia": "Africa", "Kenya": "Africa",
    "Uganda": "Africa", "Tanzania": "Africa", "Rwanda": "Africa", "Burundi": "Africa", "Democratic Republic of the Congo": "Africa",
    "Republic of the Congo": "Africa", "Angola": "Africa", "Zambia": "Africa", "Malawi": "Africa", "Mozambique": "Africa", "Zimbabwe": "Africa",
    "Botswana": "Africa", "Namibia": "Africa", "Lesotho": "Africa", "Eswatini": "Africa", "Madagascar": "Africa", "Mauritius": "Africa",
    "Comoros": "Africa", "Seychelles": "Africa", "Djibouti": "Africa", "Eritrea": "Africa", "Yemen": "Middle East", "Oman": "Middle East",
    "Qatar": "Middle East", "Bahrain": "Middle East", "Kuwait": "Middle East", "Iran": "Middle East", "Iraq": "Middle East",
    "Syria": "Middle East", "Lebanon": "Middle East", "Jordan": "Middle East", "Palestine": "Middle East", "Afghanistan": "Asia",
    "Tajikistan": "Asia", "Turkmenistan": "Asia", "Mongolia": "Asia", "Albania": "Europe", "Barbados": "North America", "Bulgaria": "Europe",
    "Cameroon": "Africa", "Croatia": "Europe", "Cyprus": "Asia", "Czechia": "Europe", "Estonia": "Europe", "Ghana": "Africa",
    "Kosovo": "Europe", "Latvia": "Europe", "Lithuania": "Europe", "Luxembourg": "Europe", "Malta": "Europe", "Moldova": "Europe",
    "Montenegro": "Europe", "North Macedonia": "Europe", "Puerto Rico": "North America", "Romania": "Europe", "Slovenia": "Europe",
    "United States of America": "North America"
}

# Apply the region mapping to create a new column
merged_df['Region'] = merged_df['Country'].map(country_to_region)

# Display the first few rows to verify the Region column
print(merged_df[['Country', 'Region']].head())

plt.figure(figsize=(12, 6))
sns.lmplot(x='Income_Efficiency_Ratio', y='Cost_of_Living_Efficiency_Ratio', hue='Region', data=merged_df, palette='pastel', aspect=1.5)
plt.title('Income Efficiency Ratio vs Cost of Living Efficiency Ratio by Region')
plt.xlabel('Income Efficiency Ratio')
plt.ylabel('Cost of Living Efficiency Ratio')
plt.show()

g = sns.FacetGrid(merged_df, col="Region", col_wrap=4, height=4, aspect=1.5)
g.map(sns.histplot, "Cost_of_Living_Efficiency_Ratio", kde=True)
g.set_titles("{col_name}")
g.set_axis_labels("Cost of Living Efficiency Ratio", "Density")
g.fig.suptitle('Cost of Living Efficiency Ratio by Region', fontsize=16)
g.fig.subplots_adjust(top=0.9)
plt.show()

     Country         Region
0    Albania         Europe
1    Algeria         Africa
2  Argentina  South America
3    Armenia           Asia
4  Australia        Oceania

<Figure size 1200x600 with 0 Axes>

# Group data by region and calculate descriptive statistics
region_grouped = merged_df.groupby('Region').agg({
    'Cost_of_Living_Efficiency_Ratio': ['mean', 'median', 'std'],
    'Income_Efficiency_Ratio': ['mean', 'median', 'std']
}).reset_index()

# Rename columns for clarity
region_grouped.columns = ['Region', 'Mean_Cost_of_Living_Efficiency_Ratio', 'Median_Cost_of_Living_Efficiency_Ratio', 'Std_Cost_of_Living_Efficiency_Ratio',
                          'Mean_Income_Efficiency_Ratio', 'Median_Income_Efficiency_Ratio', 'Std_Income_Efficiency_Ratio']

# Display the descriptive statistics
display(region_grouped)

import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

# Set the aesthetic style of the plots
sns.set(style="whitegrid")

# Scatter plot with regression line for Cost_of_Living_Efficiency_Ratio by Region using lmplot
sns.lmplot(x='Cost_of_Living_Efficiency_Ratio', y='Income_Efficiency_Ratio', data=merged_df, hue='Region', palette='pastel', height=6, aspect=1.5)
plt.title('Cost of Living Efficiency Ratio vs Income Efficiency Ratio by Region')
plt.xlabel('Cost of Living Efficiency Ratio')
plt.ylabel('Income Efficiency Ratio')
plt.xticks(rotation=45)
plt.tight_layout()
plt.show()

# Heatmap to visualize the relationship between regions and average efficiency ratios with lighter colors
plt.figure(figsize=(10, 8))
heatmap_data = merged_df.groupby('Region')[['Cost_of_Living_Efficiency_Ratio', 'Income_Efficiency_Ratio']].mean().T
sns.heatmap(heatmap_data, annot=True, cmap='YlGnBu', linewidths=0.5, cbar_kws={"shrink": .8})
plt.title('Heatmap of Average Efficiency Ratios by Region')
plt.xlabel('Region')
plt.ylabel('Efficiency Metrics')
plt.tight_layout()
plt.show()

# Pairwise relationships between efficiency ratios by region using a pair grid
g = sns.pairplot(merged_df, hue='Region', palette='bright', height=2.5, aspect=1.2,
                 vars=['Cost_of_Living_Efficiency_Ratio', 'Income_Efficiency_Ratio'])
g.fig.suptitle('Pairwise Relationships of Efficiency Ratios by Region', y=1.02)
plt.show()

from scipy import stats

# Perform ANOVA to test if there's a significant difference in the Cost of Living Efficiency Ratio across regions
anova_result_col = stats.f_oneway(*[group['Cost_of_Living_Efficiency_Ratio'].values for name, group in merged_df.groupby('Region')])
print(f"ANOVA Result for Cost of Living Efficiency Ratio across Regions: F-statistic={anova_result_col.statistic}, p-value={anova_result_col.pvalue}")

# Perform ANOVA to test if there's a significant difference in the Income Efficiency Ratio across regions
anova_result_ier = stats.f_oneway(*[group['Income_Efficiency_Ratio'].values for name, group in merged_df.groupby('Region')])
print(f"ANOVA Result for Income Efficiency Ratio across Regions: F-statistic={anova_result_ier.statistic}, p-value={anova_result_ier.pvalue}")

ANOVA Result for Cost of Living Efficiency Ratio across Regions: F-statistic=5.9413267725876375, p-value=2.763501194171203e-05
ANOVA Result for Income Efficiency Ratio across Regions: F-statistic=7.508562713436847, p-value=1.4092085055634125e-06

# Perform Kruskal-Wallis test for Cost of Living Efficiency Ratio across regions
kruskal_result_col = stats.kruskal(*[group['Cost_of_Living_Efficiency_Ratio'].values for name, group in merged_df.groupby('Region')])
print(f"Kruskal-Wallis Result for Cost of Living Efficiency Ratio across Regions: H-statistic={kruskal_result_col.statistic}, p-value={kruskal_result_col.pvalue}")

# Perform Kruskal-Wallis test for Income Efficiency Ratio across regions
kruskal_result_ier = stats.kruskal(*[group['Income_Efficiency_Ratio'].values for name, group in merged_df.groupby('Region')])
print(f"Kruskal-Wallis Result for Income Efficiency Ratio across Regions: H-statistic={kruskal_result_ier.statistic}, p-value={kruskal_result_ier.pvalue}")

Kruskal-Wallis Result for Cost of Living Efficiency Ratio across Regions: H-statistic=29.61134079238036, p-value=4.6591839535872924e-05
Kruskal-Wallis Result for Income Efficiency Ratio across Regions: H-statistic=41.20994287845775, p-value=2.632579055668303e-07

import numpy as np
import joypy

# Calculate Z-scores for Cost_of_Living_Efficiency_Ratio
merged_df['Z_Score_Cost_of_Living'] = (merged_df['Cost_of_Living_Efficiency_Ratio'] - merged_df['Cost_of_Living_Efficiency_Ratio'].mean()) / merged_df['Cost_of_Living_Efficiency_Ratio'].std()

# Calculate Z-scores for Income_Efficiency_Ratio
merged_df['Z_Score_Income'] = (merged_df['Income_Efficiency_Ratio'] - merged_df['Income_Efficiency_Ratio'].mean()) / merged_df['Income_Efficiency_Ratio'].std()

# Identify outliers (Z-scores greater than 3 or less than -3)
outliers_cost_of_living = merged_df[np.abs(merged_df['Z_Score_Cost_of_Living']) > 3]
outliers_income = merged_df[np.abs(merged_df['Z_Score_Income']) > 3]

# Display the outliers
print("Outliers based on Cost of Living Efficiency Ratio Z-Scores:")
print(outliers_cost_of_living[['Country', 'Z_Score_Cost_of_Living']])

print("\nOutliers based on Income Efficiency Ratio Z-Scores:")
print(outliers_income[['Country', 'Z_Score_Income']])

# Ridgeline plot for Cost_of_Living_Efficiency_Ratio by Region
plt.figure(figsize=(12, 8))
joypy.joyplot(data=merged_df, by='Region', column='Cost_of_Living_Efficiency_Ratio', 
              ylim='own', colormap=plt.cm.Pastel1, fade=True)
plt.title('Ridgeline Plot of Cost of Living Efficiency Ratio by Region')
plt.xlabel('Cost of Living Efficiency Ratio')
plt.show()

# Ridgeline plot for Income_Efficiency_Ratio by Region
plt.figure(figsize=(12, 8))
joypy.joyplot(data=merged_df, by='Region', column='Income_Efficiency_Ratio', 
              ylim='own', colormap=plt.cm.Pastel2, fade=True)
plt.title('Ridgeline Plot of Income Efficiency Ratio by Region')
plt.xlabel('Income Efficiency Ratio')
plt.show()

Outliers based on Cost of Living Efficiency Ratio Z-Scores:
     Country  Z_Score_Cost_of_Living
17  Cameroon                5.651787
71   Nigeria                4.210824

Outliers based on Income Efficiency Ratio Z-Scores:
       Country  Z_Score_Income
59  Luxembourg        3.155711

<Figure size 1200x800 with 0 Axes>

<Figure size 1200x800 with 0 Axes>

import pandas as pd
import numpy as np

# Load datasets
cost_of_living_df = pd.read_csv(r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\Cost_of_Living_Index_by_Country_2024.csv')
gdp_df = pd.read_csv(r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\Economic Analysis data\GDP per capita (current US$\GDP 2023.csv')
gni_df = pd.read_csv(r'C:\Users\eskbe\OneDrive\Desktop\Esk\August 17 Data Science Project\Cost of Living Index\Economic Analysis data\GNI per capita, PPP (current international $)\GNI 2023.csv')

# Rename columns to avoid confusion
gdp_df.rename(columns={'2023': 'GDP_2023'}, inplace=True)
gni_df.rename(columns={'2023': 'GNI_2023'}, inplace=True)

# Merge datasets on 'Country'
merged_df = cost_of_living_df.merge(gdp_df, on='Country', how='inner')
merged_df = merged_df.merge(gni_df, on='Country', how='inner')

# Drop any rows with missing data
merged_df.dropna(inplace=True)

# Add basic economic efficiency ratios
merged_df['Cost_of_Living_Efficiency_Ratio'] = merged_df['Cost of Living Index'] / merged_df['Local Purchasing Power Index']
merged_df['Income_Efficiency_Ratio'] = merged_df['GDP_2023'] / merged_df['GNI_2023']

# Define scenario variables (Example: High Inflation Scenario)
def apply_inflation_scenario(df, inflation_rate):
    df['Adjusted_Cost_of_Living_Index'] = df['Cost of Living Index'] * (1 + inflation_rate)
    df['Adjusted_Cost_of_Living_Efficiency_Ratio'] = df['Adjusted_Cost_of_Living_Index'] / df['Local Purchasing Power Index']
    return df

# Example: High inflation scenario (+10% inflation)
high_inflation_df = apply_inflation_scenario(merged_df.copy(), 0.10)

# Monte Carlo Simulation Example: Vary inflation rates and calculate efficiency ratios
def monte_carlo_simulation(df, n_simulations=1000):
    results = []
    for _ in range(n_simulations):
        inflation_rate = np.random.uniform(0.05, 0.15)  # Vary inflation between 5% and 15%
        scenario_df = apply_inflation_scenario(df.copy(), inflation_rate)
        results.append(scenario_df[['Country', 'Adjusted_Cost_of_Living_Efficiency_Ratio']])
    return pd.concat(results, axis=0)

# Run the Monte Carlo simulation
simulation_results = monte_carlo_simulation(merged_df)

# Display some results
print(simulation_results.head())

# Sensitivity Analysis: Varying one input while keeping others constant
def sensitivity_analysis(df, variable, min_change, max_change, step):
    sensitivity_results = []
    for change in np.arange(min_change, max_change, step):
        if variable == 'inflation_rate':
            scenario_df = apply_inflation_scenario(df.copy(), change)
            sensitivity_results.append(scenario_df[['Country', 'Adjusted_Cost_of_Living_Efficiency_Ratio']])
    return pd.concat(sensitivity_results, axis=0)

# Run sensitivity analysis for inflation rate
sensitivity_results = sensitivity_analysis(merged_df, 'inflation_rate', 0.05, 0.15, 0.01)

# Display sensitivity analysis results
print(sensitivity_results.head())

     Country  Adjusted_Cost_of_Living_Efficiency_Ratio
0    Albania                                  1.187777
1    Algeria                                  1.088059
2  Argentina                                  0.797489
3    Armenia                                  1.198806
4  Australia                                  0.620288
     Country  Adjusted_Cost_of_Living_Efficiency_Ratio
0    Albania                                  1.107895
1    Algeria                                  1.014883
2  Argentina                                  0.743855
3    Armenia                                  1.118182
4  Australia                                  0.578571

import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np

# Assuming the data has already been merged and cleaned in previous steps
# We'll adjust the Cost of Living Index and Local Purchasing Power Index to simulate improved trade agreements

# Define adjustment factors based on the scenario
cost_of_living_reduction_factor = 0.9  # 10% reduction in Cost of Living Index
income_increase_factor = 1.1           # 10% increase in Local Purchasing Power Index

# Apply the adjustments to the Cost of Living Index and Local Purchasing Power Index
merged_df['Adjusted_Cost_of_Living_Index'] = merged_df['Cost of Living Index'] * cost_of_living_reduction_factor
merged_df['Adjusted_Local_Purchasing_Power_Index'] = merged_df['Local Purchasing Power Index'] * income_increase_factor

# Recalculate the efficiency ratios with the adjusted values
merged_df['Adjusted_Cost_of_Living_Efficiency_Ratio'] = merged_df['Adjusted_Cost_of_Living_Index'] / merged_df['Adjusted_Local_Purchasing_Power_Index']
merged_df['Adjusted_Income_Efficiency_Ratio'] = merged_df['GDP_2023'] / merged_df['Adjusted_Local_Purchasing_Power_Index']

# Display the first few rows to check the new efficiency ratios
print(merged_df[['Country', 'Adjusted_Cost_of_Living_Efficiency_Ratio', 'Adjusted_Income_Efficiency_Ratio']].head())

# Summarize the results
summary = merged_df[['Region', 'Adjusted_Cost_of_Living_Efficiency_Ratio', 'Adjusted_Income_Efficiency_Ratio']].groupby('Region').agg(['mean', 'median', 'std'])
print(summary)

# Optionally, you can visualize the results
sns.set(style="whitegrid")

# Visualize the adjusted Cost of Living Efficiency Ratio by Region
plt.figure(figsize=(12, 6))
sns.boxplot(x='Region', y='Adjusted_Cost_of_Living_Efficiency_Ratio', hue='Region', data=merged_df, palette='pastel', dodge=False)
plt.title('Adjusted Cost of Living Efficiency Ratio by Region (Improved Trade Agreements)')
plt.xlabel('Region')
plt.ylabel('Adjusted Cost of Living Efficiency Ratio')
plt.xticks(rotation=45)
plt.legend([],[], frameon=False)  # Hides the legend created by hue
plt.tight_layout()
plt.show()

# Visualize the adjusted Income Efficiency Ratio by Region
plt.figure(figsize=(12, 6))
sns.boxplot(x='Region', y='Adjusted_Income_Efficiency_Ratio', hue='Region', data=merged_df, palette='pastel', dodge=False)
plt.title('Adjusted Income Efficiency Ratio by Region (Improved Trade Agreements)')
plt.xlabel('Region')
plt.ylabel('Adjusted Income Efficiency Ratio')
plt.xticks(rotation=45)
plt.legend([],[], frameon=False)  # Hides the legend created by hue
plt.tight_layout()
plt.show()

     Country  Adjusted_Cost_of_Living_Efficiency_Ratio  \
0    Albania                                  0.863295   
1    Algeria                                  0.790818   
2  Argentina                                  0.579628   
3    Armenia                                  0.871311   
4  Australia                                  0.450835   

   Adjusted_Income_Efficiency_Ratio  
0                        190.653355  
1                        159.933300  
2                        300.777978  
3                        205.803196  
4                        461.765132  
              Adjusted_Cost_of_Living_Efficiency_Ratio                      \
                                                  mean    median       std   
Region                                                                       
Africa                                        1.072457  0.756653  0.754968   
Asia                                          0.667100  0.603523  0.340534   
Europe                                        0.523975  0.516839  0.115725   
Middle East                                   0.404110  0.382244  0.173583   
North America                                 0.888166  0.920158  0.335476   
Oceania                                       0.437862  0.436814  0.012483   
South America                                 0.664619  0.664223  0.067304   

              Adjusted_Income_Efficiency_Ratio                          
                                          mean      median         std  
Region                                                                  
Africa                              106.593574  106.158756   57.565005  
Asia                                180.785824  149.064296  163.918329  
Europe                              368.763630  361.428379  176.447282  
Middle East                         231.551502  202.803793  135.210287  
North America                       329.974268  326.551956  129.135795  
Oceania                             301.604406  364.596761  199.269657  
South America                       226.994493  203.497152   93.959966

import numpy as np

# Define a function to simulate the recession impact
def apply_recession_impact(df, recession_countries, gdp_reduction=0.15, income_reduction=0.20, inflation_increase=0.10):
    df_copy = df.copy()
    
    # Apply GDP reduction
    df_copy.loc[df_copy['Country'].isin(recession_countries), 'GDP_2023'] *= (1 - gdp_reduction)
    
    # Apply Income reduction
    df_copy.loc[df_copy['Country'].isin(recession_countries), 'GNI_2023'] *= (1 - income_reduction)
    
    # Apply Inflation increase by adjusting Cost of Living Index
    df_copy.loc[df_copy['Country'].isin(recession_countries), 'Cost_of_Living_Efficiency_Ratio'] *= (1 + inflation_increase)
    
    # Recalculate the efficiency ratios
    df_copy['Adjusted_Cost_of_Living_Efficiency_Ratio'] = df_copy['Cost_of_Living_Efficiency_Ratio'] / df_copy['Local Purchasing Power Index']
    df_copy['Adjusted_Income_Efficiency_Ratio'] = df_copy['GDP_2023'] / df_copy['GNI_2023']
    
    return df_copy

# Define the major economies likely to be impacted by a recession
recession_countries = ['United States', 'China', 'Germany']

# Apply the recession scenario to the merged data
recession_df = apply_recession_impact(merged_df, recession_countries)

# Display a few rows to verify
print(recession_df[['Country', 'Adjusted_Cost_of_Living_Efficiency_Ratio', 'Adjusted_Income_Efficiency_Ratio']].head())

     Country  Adjusted_Cost_of_Living_Efficiency_Ratio  \
0    Albania                                  0.026445   
1    Algeria                                  0.032326   
2  Argentina                                  0.017071   
3    Armenia                                  0.027661   
4  Australia                                  0.004325   

   Adjusted_Income_Efficiency_Ratio  
0                          0.396389  
1                          0.313294  
2                          0.478249  
3                          0.388403  
4                          0.976634

import matplotlib.pyplot as plt
import seaborn as sns

# Create scenario DataFrames from merged_df
inflation_df = merged_df.copy()
trade_df = merged_df.copy()
recession_df = merged_df.copy()

# Apply example transformations
inflation_df['Adjusted_Cost_of_Living_Efficiency_Ratio'] = inflation_df['Cost_of_Living_Efficiency_Ratio'] * 1.2
trade_df['Adjusted_Cost_of_Living_Efficiency_Ratio'] = trade_df['Cost_of_Living_Efficiency_Ratio'] * 0.8
recession_df['Adjusted_Cost_of_Living_Efficiency_Ratio'] = recession_df['Cost_of_Living_Efficiency_Ratio'] * 0.5

# Visualization code
plt.figure(figsize=(18, 12))

# Set the font scale for the seaborn context
sns.set_context("talk")  # You can use 'paper', 'notebook', 'talk', or 'poster'

plt.figure(figsize=(20, 12))

# Heatmap for Original Scenario
plt.subplot(2, 2, 1)
heatmap_data_original = merged_df.pivot(index="Country", columns="Region", values="Cost_of_Living_Efficiency_Ratio")
sns.heatmap(heatmap_data_original, cmap="YlGnBu", annot=True, fmt=".2f", linewidths=0.5, annot_kws={"size": 10})
plt.title('Original Cost of Living Efficiency Ratio', fontsize=16)
plt.xticks(rotation=45, fontsize=12)
plt.yticks(fontsize=12)

# Heatmap for High Inflation Scenario
plt.subplot(2, 2, 2)
heatmap_data_inflation = inflation_df.pivot(index="Country", columns="Region", values="Adjusted_Cost_of_Living_Efficiency_Ratio")
sns.heatmap(heatmap_data_inflation, cmap="YlGnBu", annot=True, fmt=".2f", linewidths=0.5, annot_kws={"size": 10})
plt.title('High Inflation Impact on Cost of Living Efficiency Ratio', fontsize=16)
plt.xticks(rotation=45, fontsize=12)
plt.yticks(fontsize=12)

# Heatmap for Improved Trade Agreements Scenario
plt.subplot(2, 2, 3)
heatmap_data_trade = trade_df.pivot(index="Country", columns="Region", values="Adjusted_Cost_of_Living_Efficiency_Ratio")
sns.heatmap(heatmap_data_trade, cmap="YlGnBu", annot=True, fmt=".2f", linewidths=0.5, annot_kws={"size": 10})
plt.title('Improved Trade Agreements Impact on Cost of Living Efficiency Ratio', fontsize=16)
plt.xticks(rotation=45, fontsize=12)
plt.yticks(fontsize=12)

# Heatmap for Recession Scenario
plt.subplot(2, 2, 4)
heatmap_data_recession = recession_df.pivot(index="Country", columns="Region", values="Adjusted_Cost_of_Living_Efficiency_Ratio")
sns.heatmap(heatmap_data_recession, cmap="YlGnBu", annot=True, fmt=".2f", linewidths=0.5, annot_kws={"size": 10})
plt.title('Recession Impact on Cost of Living Efficiency Ratio', fontsize=16)
plt.xticks(rotation=45, fontsize=12)
plt.yticks(fontsize=12)

plt.tight_layout()
plt.show()


# Line Chart Comparison
plt.figure(figsize=(12, 8))
sns.lineplot(x="Region", y="Cost_of_Living_Efficiency_Ratio", data=merged_df, label='Original', marker='o')
sns.lineplot(x="Region", y="Adjusted_Cost_of_Living_Efficiency_Ratio", data=inflation_df, label='High Inflation', marker='o')
sns.lineplot(x="Region", y="Adjusted_Cost_of_Living_Efficiency_Ratio", data=trade_df, label='Improved Trade', marker='o')
sns.lineplot(x="Region", y="Adjusted_Cost_of_Living_Efficiency_Ratio", data=recession_df, label='Recession', marker='o')
plt.title('Comparison of Cost of Living Efficiency Ratios Across Scenarios')
plt.xlabel('Region')
plt.ylabel('Cost of Living Efficiency Ratio')
plt.xticks(rotation=45)
plt.legend()
plt.tight_layout()
plt.show()

<Figure size 1800x1200 with 0 Axes>

import pandas as pd

# Create a summary table for each scenario
summary_table = pd.DataFrame({
    'Scenario': ['Original', 'High Inflation', 'Improved Trade', 'Recession'],
    'Mean Cost of Living Efficiency Ratio': [
        merged_df['Cost_of_Living_Efficiency_Ratio'].mean(),
        inflation_df['Adjusted_Cost_of_Living_Efficiency_Ratio'].mean(),
        trade_df['Adjusted_Cost_of_Living_Efficiency_Ratio'].mean(),
        recession_df['Adjusted_Cost_of_Living_Efficiency_Ratio'].mean()
    ],
    'Median Cost of Living Efficiency Ratio': [
        merged_df['Cost_of_Living_Efficiency_Ratio'].median(),
        inflation_df['Adjusted_Cost_of_Living_Efficiency_Ratio'].median(),
        trade_df['Adjusted_Cost_of_Living_Efficiency_Ratio'].median(),
        recession_df['Adjusted_Cost_of_Living_Efficiency_Ratio'].median()
    ],
    'Std Dev Cost of Living Efficiency Ratio': [
        merged_df['Cost_of_Living_Efficiency_Ratio'].std(),
        inflation_df['Adjusted_Cost_of_Living_Efficiency_Ratio'].std(),
        trade_df['Adjusted_Cost_of_Living_Efficiency_Ratio'].std(),
        recession_df['Adjusted_Cost_of_Living_Efficiency_Ratio'].std()
    ],
    'Mean Income Efficiency Ratio': [
        merged_df['Income_Efficiency_Ratio'].mean(),
        inflation_df['Adjusted_Income_Efficiency_Ratio'].mean(),
        trade_df['Adjusted_Income_Efficiency_Ratio'].mean(),
        recession_df['Adjusted_Income_Efficiency_Ratio'].mean()
    ],
    'Median Income Efficiency Ratio': [
        merged_df['Income_Efficiency_Ratio'].median(),
        inflation_df['Adjusted_Income_Efficiency_Ratio'].median(),
        trade_df['Adjusted_Income_Efficiency_Ratio'].median(),
        recession_df['Adjusted_Income_Efficiency_Ratio'].median()
    ],
    'Std Dev Income Efficiency Ratio': [
        merged_df['Income_Efficiency_Ratio'].std(),
        inflation_df['Adjusted_Income_Efficiency_Ratio'].std(),
        trade_df['Adjusted_Income_Efficiency_Ratio'].std(),
        recession_df['Adjusted_Income_Efficiency_Ratio'].std()
    ]
})

# Display the summary table
display(summary_table)

	Rank	Cost of Living Index	Rent Index	Cost of Living Plus Rent Index	Groceries Index	Restaurant Price Index	Local Purchasing Power Index
count	121.000000	121.000000	121.000000	121.000000	121.000000	121.000000	121.000000
mean	61.000000	43.555372	16.052893	30.357851	44.228926	36.471074	65.094215
std	35.073732	16.147574	11.412267	13.263721	17.055109	18.258110	39.569094
min	1.000000	18.800000	2.400000	11.100000	17.500000	12.800000	2.300000
25%	31.000000	30.200000	8.500000	19.800000	31.600000	21.600000	34.800000
50%	61.000000	39.500000	12.400000	27.000000	40.500000	33.100000	50.600000
75%	91.000000	52.800000	20.100000	37.000000	53.700000	47.200000	99.400000
max	121.000000	101.100000	67.200000	74.900000	109.100000	97.000000	182.500000

	Cost of Living Index	Rent Index	Cost of Living Plus Rent Index	Groceries Index	Restaurant Price Index	Local Purchasing Power Index
Cost of Living Index	1.000000	0.820885	0.971780	0.958452	0.945483	0.692688
Rent Index	0.820885	1.000000	0.932425	0.770944	0.763537	0.683912
Cost of Living Plus Rent Index	0.971780	0.932425	1.000000	0.924935	0.913618	0.720701
Groceries Index	0.958452	0.770944	0.924935	1.000000	0.855057	0.640634
Restaurant Price Index	0.945483	0.763537	0.913618	0.855057	1.000000	0.673539
Local Purchasing Power Index	0.692688	0.683912	0.720701	0.640634	0.673539	1.000000

	Cost of Living Index	Rent Index	Cost of Living Plus Rent Index	Groceries Index	Restaurant Price Index	Local Purchasing Power Index
Cost of Living Index	0.000000e+00	9.894907e-31	1.687787e-76	1.132141e-66	8.094302e-60	1.349649e-18
Rent Index	9.894907e-31	0.000000e+00	1.943347e-54	4.555238e-25	2.380434e-24	5.356191e-18
Cost of Living Plus Rent Index	1.687787e-76	1.943347e-54	0.000000e+00	8.086503e-52	2.451265e-48	1.169151e-20
Groceries Index	1.132141e-66	4.555238e-25	8.086503e-52	0.000000e+00	9.739186e-36	2.507723e-15
Restaurant Price Index	8.094302e-60	2.380434e-24	2.451265e-48	9.739186e-36	0.000000e+00	2.570035e-17
Local Purchasing Power Index	1.349649e-18	5.356191e-18	1.169151e-20	2.507723e-15	2.570035e-17	0.000000e+00

	PC1	PC2
Cost of Living Index	0.432645	-0.228391
Rent Index	0.397578	0.092917
Cost of Living Plus Rent Index	0.437963	-0.106309
Groceries Index	0.413752	-0.299175
Restaurant Price Index	0.413335	-0.191430
Local Purchasing Power Index	0.347708	0.895405

	Region	Mean_Cost_of_Living_Efficiency_Ratio	Median_Cost_of_Living_Efficiency_Ratio	Std_Cost_of_Living_Efficiency_Ratio	Mean_Income_Efficiency_Ratio	Median_Income_Efficiency_Ratio	Std_Income_Efficiency_Ratio
0	Africa	1.310780	0.924798	0.922738	0.339172	0.325602	0.047455
1	Asia	0.815345	0.737639	0.416209	0.376276	0.319268	0.153834
2	Europe	0.640414	0.631692	0.141441	0.631109	0.594190	0.241886
3	Middle East	0.493912	0.467187	0.212158	0.570338	0.538383	0.182087
4	North America	1.085537	1.124638	0.410026	0.711761	0.615968	0.290521
5	Oceania	0.535164	0.533884	0.015256	0.771916	0.919959	0.306812
6	South America	0.812312	0.811828	0.082260	0.466122	0.478249	0.118277

	Scenario	Mean Cost of Living Efficiency Ratio	Median Cost of Living Efficiency Ratio	Std Dev Cost of Living Efficiency Ratio	Mean Income Efficiency Ratio	Median Income Efficiency Ratio	Std Dev Income Efficiency Ratio
0	Original	0.815309	0.684850	0.484284	0.537753	0.472637	0.242261
1	High Inflation	0.978371	0.821820	0.581141	268.206018	222.590261	174.487771
2	Improved Trade	0.652247	0.547880	0.387427	268.206018	222.590261	174.487771
3	Recession	0.407655	0.342425	0.242142	268.206018	222.590261	174.487771

Global Economic Efficiency and Cost of Living Analysis 2024: A Comprehensive Study Using Multivariate and Geospatial Analytics

Introduction¶

Background and Analysis¶

Table of Contents¶

1. Data Overview¶

2. Data Import and Preparation¶

3. Exploratory Data Analysis (EDA)¶

3.1 Distribution of Indices¶

3.2 Pairwise Relationships¶

3.3 Correlation Matrix¶

Summary of Findings from Exploratory Data Analysis (EDA):¶

4. Correlation Analysis¶

4.1 Detailed Correlation Analysis¶

4.2 Statistical Significance Testing¶

4.3 Interpretation of Results¶

5. Cluster Analysis¶

5.1 K-Means Clustering¶

5.2 Applying K-Means Clustering¶

Summary of Cluster Analysis¶

6. Principal Component Analysis (PCA)¶

6.1 Applying PCA¶

6.2 Interpreting PCA Results¶

6.3 Visualizing Explained Variance¶

Summary of Principal Component Analysis (PCA)¶

7. Regression Analysis¶

7.1 Simple Linear Regression¶

7.2 Multiple Linear Regression¶

7.3 Regression Diagnostics¶

Conclusion of Regression Analysis¶

8. Geospatial Analysis¶

8.1 Mapping Economic Indices¶

8.2 Heatmap of Economic Hotspots¶

8.3 Regional Analysis¶

Geospatial Analysis Summary¶

9. Machine Learning¶

9.1 Data Preparation for Machine Learning¶

9.2 Model Selection¶

9.3 Model Evaluation and Tuning¶

9.4 Model Interpretation and Insights¶

9.5 Refining Machine Learning Model¶

Summary of Machine Learning Models¶

10. Hypothesis Testing¶

10.1 Formulating Hypotheses¶

Question 1: Is there a significant difference in the average Cost of Living Index between different regions?¶

Question 2: Is there a significant correlation between the Cost of Living Index and the Local Purchasing Power Index?¶

Question 3: Is there a significant difference in economic indices between high-income and low-income countries?¶

10.2 Choosing Appropriate Tests¶

Hypothesis 1: Is there a significant difference in the average Cost of Living Index between different regions?¶

Hypothesis 2: Is there a significant correlation between the Cost of Living Index and the Local Purchasing Power Index?¶

Hypothesis 3: Is there a significant difference in economic indices between high-income and low-income countries?¶

10.2.1. Check Assumptions¶

a. Normality (for ANOVA and T-Test): Shapiro-Wilk Test and Q-Q Plot¶

b. Homogeneity of Variances (for ANOVA): Levene's Test¶

c. Linearity (for Pearson Correlation): Scatter Plot:¶

Assumptions and Justifications¶

10.3 Performing Hypothesis Tests¶

Hypothesis Test Results¶

10.4 Interpreting Results¶

Interpretation of Hypothesis 1: Regional Differences in Cost of Living Index¶

Interpretation of Hypothesis 2: Correlation Between Cost of Living Index and Local Purchasing Power Index¶

Interpretation of Hypothesis 3: Differences Between High-Income and Low-Income Countries¶

Summary of Findings¶

11. Economic Efficiency Analysis¶

11.1 Define Economic Efficiency Metrics¶

Data Preparation:¶

11.2 Calculate Efficiency Ratios¶

11.2.1 Descriptive Statistics¶

11.2.2 Identify Top and Bottom Countries¶

11.2.2 Visual Analysis¶

Histograms for Efficiency Ratios¶

Box Plots to Identify Outliers¶

Regional Comparison¶

11.3 Compare Economic Efficiency Across Regions¶

11.3.1 Descriptive Statistics by Region¶

11.3.2 Visualization¶

11.3.3 ANOVA Test¶

11.3.4 Kruskal-Wallis Test¶

11.4 Identify Outliers and Exceptional Performers¶

11..4.1 Ridgeline Plot Analysis and Z-Score Calculation¶

In-Depth Analysis of Outliers¶

1. Data Overview ¶

2. Data Import and Preparation ¶

3.1 Distribution of Indices ¶

3.2 Pairwise Relationships ¶

3.3 Correlation Matrix ¶

4. Correlation Analysis ¶

4.1 Detailed Correlation Analysis ¶

4.2 Statistical Significance Testing ¶

4.3 Interpretation of Results ¶

5. Cluster Analysis ¶

5.1 K-Means Clustering ¶

5.2 Applying K-Means Clustering ¶

6.1 Applying PCA ¶

6.2 Interpreting PCA Results ¶

6.3 Visualizing Explained Variance ¶

7. Regression Analysis ¶

7.1 Simple Linear Regression ¶

7.2 Multiple Linear Regression ¶

7.3 Regression Diagnostics ¶

8. Geospatial Analysis ¶

8.1 Mapping Economic Indices ¶

8.2 Heatmap of Economic Hotspots ¶

8.3 Regional Analysis ¶

9. Machine Learning ¶

9.1 Data Preparation for Machine Learning ¶

9.2 Model Selection ¶

9.3 Model Evaluation and Tuning ¶

9.4 Model Interpretation and Insights ¶

9.5 Refining Machine Learning Model ¶

10. Hypothesis Testing ¶

10.1 Formulating Hypotheses ¶

10.2 Choosing Appropriate Tests ¶

10.3 Performing Hypothesis Tests ¶

10.4 Interpreting Results ¶

11. Economic Efficiency Analysis ¶

11.1 Define Economic Efficiency Metrics ¶

11.2 Calculate Efficiency Ratios ¶

11.3 Compare Economic Efficiency Across Regions ¶

11.4 Identify Outliers and Exceptional Performers ¶

11.5 Interpretation of Results ¶

12. Scenario Analysis and Simulations ¶

13. Conclusion ¶